#!/bin/sh

export LANG=c

site=dangdang
queue=${site}_queue
savedir=/data/${site}/

mkdir -p $savedir
rm -f $queue

page=0
explore=./.explore_${site}.bdb
rm -f $explore

while [ 1 ]
do
	page=$(($page+1))

	./neoparse dangdang_product.xml \
	"http://cosmetic.dangdang.com/" > /tmp/$queue
	#"http://category.dangdang.com/list?ps=28&cat=4002074&sort=5&store=eq0&p=${page}" > /tmp/$queue
 
	grep "link: " /tmp/$queue | grep -i "Product.aspx?product_id=" | awk '{print $2}' > $queue
	cp $queue /tmp/$queue.$page

	python neoexplore.py $queue $explore
	if [ $? -ne 0 ]
	then
		break
	fi

	python neospider.py $queue $savedir
	sleep 2
done

